﻿using System.Threading;
using DM4;
using sscore;
using System;
using System.Collections.Generic;
using sscore.Utils;
using System.Text.RegularExpressions;
using System.IO;
using sscore.Requests;

namespace IA
{
    public class IA : ModuleBase
    {
        private readonly string _searchUrl;
        private readonly string _offenderLink;
        private const string StateName = "IA";
        private ResultFile _imagesResultFile;
        readonly string[] _searchParam = { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };

        public IA()
        {
            //initialize main system constants
            ModuleCode = "SO_IA";
            Description = "IA Sex Offenders";
            Version = "1.0";

            //for creating sessions
            StartParam = 0;
            StopParam = _searchParam.Length - 1;

            //initialize help 
            AddToHelp(DM4Consts.StartPosition, String.Format("{0}...{1}", StartParam, StopParam));
            AddToHelp(DM4Consts.EndPositon, String.Format("{0}...{1}", StartParam, StopParam));

            AddToHelp(DM4Consts.IsDownload, "true|false");
            AddToHelp(DM4Consts.IsDownloadImg, "true|false");
            AddToHelp(DM4Consts.InsertToDb, "false|true", "(Expected: servername, dbname)");
            AddToHelp(DM4Consts.ServerName, "");
            AddToHelp(DM4Consts.DbName, "");
            AddToHelp(DM4Consts.IsClearDb, "false|true");
            AddToHelp(DM4Consts.TestParam, "false|true");

            RootUrl = "http://www.iowasexoffender.com/";
            _searchUrl = String.Format("{0}simplesearch/?lastname={{0}}&type=simple&ResultsPaging-%3ESortBy=last_name&ResultsPaging-%3EPage={{1}}#Results", RootUrl);
            _offenderLink = string.Format("{0}registrant/{{0}}/", RootUrl);
        }

        private void SetParametersDefaults()
        {
            Params.CreateBooleanParam(DM4Consts.IsDownload, true);
            Params.CreateBooleanParam(DM4Consts.IsDownloadImg, true);
            Params.CreateBooleanParam(DM4Consts.InsertToDb, false);
            Params.CreateBooleanParam(DM4Consts.IsClearDb, false);
            Params.CreateStringParam(DM4Consts.DbName, "");
            Params.CreateStringParam(DM4Consts.ServerName, "");
            Params.CreateIntegerParam(DM4Consts.StartPosition, StartParam);
            Params.CreateIntegerParam(DM4Consts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
            Params.CreateIntegerParam(DM4Consts.EndPositon, StopParam);
            Params.CreateIntegerParam(DM4Consts.LastRecord, 0);
            Params.CreateIntegerParam(DmConsts.LastPage, 1);
        }

        public override OperationResult Init(IDM4Core iCore, DmTaskParams param)
        {
            //use basic initialization sequence
            base.Init(iCore, param);

            SetParametersDefaults();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult Start()
        {
            DoWork();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult DoWork()
        {
            OperationResult or = new OperationResult { Result = DM4Consts.ErrorUnknown };

            if (!Ct.IsCancellationRequested)
            {
                WriteLog("Start parameters:", MessageType.Warning);
                foreach (KeyValuePair<string, string> kvp in Params.GetDictionary()) //get all existing parameters
                    WriteLog(String.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value), MessageType.Warning);

                if (Params.GetBooleanParam(DM4Consts.IsDownload))
                    Load();

                if (Params.GetBooleanParam(DM4Consts.InsertToDb))
                    if ((Params.GetStringParam(DM4Consts.DbName) == "") || (Params.GetStringParam(DM4Consts.ServerName) == ""))
                    {
                        or.Message = String.Format("{0} or {1} is empty", DM4Consts.DbName, DM4Consts.ServerName);
                        ResultCode = DM4Consts.SessionResultCodeError;
                        ResultMessage = or.Message;
                        UpdateStatus(DmConsts.Error);
                        WriteErrorLog(or.Message); //writting to error and session log   

                        return or;
                    }
                    else
                    {
                        //insert offenders to db 
                        UpdateStatus(DM4Consts.SessionInsertToDb);

                        Offenders.OffInsertToDatabase(StateName, Params.GetStringParam(DmConsts.DbName),
                             Params.GetStringParam(DmConsts.ServerName), ResultDir, Params.GetBooleanParam(DmConsts.IsClearDb),
                             WriteLog);

                    }

                //check if need abort task
                if (Ct.IsCancellationRequested)
                {
                    ResultCode = DM4Consts.SessionResultTerminateByUser;
                    ResultMessage = "Session terminated by user";
                    UpdateStatus(DM4Consts.SessionWait);
                    WriteLog("Session aborted by user", MessageType.Cool);
                }
                else
                {
                    UpdateProgress(1);
                    ResultCode = DM4Consts.SessionResultCodeOk;
                    ResultMessage = String.Format("Session complete, errors : {0}, records : {1}", TaskParams.ErrorCount, TaskParams.RecordCount);
                    UpdateStatus(DM4Consts.SessionComplete);
                    WriteLog("------------Finish----------------", MessageType.Cool);
                }
            }
            else
            {
                ResultCode = DM4Consts.SessionResultTerminateByUser;
                ResultMessage = "Session terminated by user";
                UpdateStatus(DM4Consts.SessionWait);
                WriteLog("Session aborted by user", MessageType.Cool);
            }


            return or;
        }

        private void Load()
        {
            string resultFileName = String.Format("{0}\\{1}.{2}", ResultDir, TaskParams.SessionId, DmConsts.ResultFilesExtension);
            bool workFlag = true;

            ResultFile = new ResultFile(resultFileName, OffendersResult.GetHeader());
            _imagesResultFile = new ResultFile(String.Format("{0}\\{1}_images.{2}", ResultDir, TaskParams.SessionId, DM4Consts.ResultFilesExtension), "FileName|FileUrl");
            try
            {
                while (workFlag)
                {
                    //рабочий цикл
                    string error;

                    try
                    {
                        error = "";  //Required!!  /*if (!String.IsNullOrEmpty(error)) --> restart thread; else --> download finished*/

                        WriteLog("Start download...");
                        Searching();
                    }
                    catch (Exception e)
                    {
                        error = e.Message;
                    }

                    if (!String.IsNullOrEmpty(error))
                    {
                        Thread.Sleep(30000);
                        WriteErrorLog(error);
                        WriteLog("Try continue work...", MessageType.Warning);
                    }
                    else
                    {
                        workFlag = false;  //все нормально откачало, заканчиваем работу
                    }
                }
            }
            finally
            {
                ResultFile.Close();
                _imagesResultFile.Close();
            }
        }

        private string GetCookies()
        {
            WriteLog("Get cookies");
            string html = GetRequest.ReturnCookies(RootUrl + "search", Rpm);
            Rpm.Cookies = RegExpr.Parse(Rpm.Cookies, "(?si:(PHPSESSID=[^;]+;))") + "COOKIES_ENABLED=1";

            string captchaId = RegExpr.Parse(html, @"(?si:captcha\x3Fsid=([^""]+)"")");
            string captchaPath = String.Format("{0}\\{1}\\{2}.jpg", ResultDir, DmConsts.PhotosDir, Guid.NewGuid());

            WriteLog("get captcha");
            string cookies = Rpm.Cookies;
            ImageUtils.DownloadImage(captchaPath, String.Format("http://www.iowasexoffender.com/captcha?sid={0}", captchaId), ref cookies, "", "", "80", true, true, TaskParams.SessionId);

            WriteLog("get captcha response");
            string captchaResp = ImageUtils.CrackCaptcha(captchaPath, false, Consts.CaptchaKeyIgor, "", 80, "", true, true);
            WriteLog("captcha response: " + captchaResp);
            WriteLog("accept disclaimer");
            html = PostRedirectRequest.RefererWithCookies(
                string.Format("{0}accept/", RootUrl),
                string.Format("{0}search", RootUrl),
                string.Format("agreement=%0D%0A%3D%3D%3D+PUBLIC+NOTICE+AND+DISCLAIMER+%3D%3D%3D%0D%0A%0D%0A1%29+This+site+does+not+contain+the+entire+list+of+sex+offenders+registered+in+Iowa.%0D%0A%0D%0A2%29+Your+community+law+enforcement+and+county+sheriff%27s+office+are+aware+that+these+subjects+are+in+the+community.%0D%0A%0D%0A3%29+Any+actions+taken+by+you+against+these+subjects%2C+including+vandalism+of+property%2C+verbal+or+written+threats+of+harm+or+physical+assault+against+these+subjects%2C+their+families+or+employers+can+result+in+your+arrest+and+prosecution.%0D%0A%0D%0A4%29+You+must+contact+your+police+department+or+Sheriff%27s+office+immediately+if+you+believe+a+crime+is+being%2C+or+will+be%2C+committed.+If+you+have+any+questions+regarding+this+matter%2C+contact+your+local+police+department+or+county+sheriff%27s+office.%0D%0A%0D%0A5%29+Scripted+or+automated+access+to+this+site+is+denied+outside+of+the+site%27s+rss+feed+and+api.+Do+not+scrape+or+crawl+this+site%27s+data%2C+the+api+offered+will+assure+correctness.%0D%0A%0D%0A6%29+Registrants+are+mapped+to+the+best+ability+with+the+data+provided%2C+DPS%2FDCI+makes+no+guarantee+of+the+validity+of+the+mapped+locations+of+registrants.%0D%0A%0D%0A%3D%3D%3D+DISCLAIMER+-+TERMS+AND+CONDITIONS+%3D%3D%3D%0D%0A%0D%0AThe+State+of+Iowa+and+its+agencies%2C+officials+and+employees+make+no+warranty%2C+representation+or+guaranty+as+to+the+content%2C+accuracy%2C+timeliness+or+completeness+of+the+information+provided+herein.+The+State+of+Iowa%2C+and+the+Department+of+Administrative+Services%2C+Information+Technology+Enterprise+expressly+disclaim+any+and+all+liability+for+any+loss+or+injury+caused%2C+in+whole+or+in+part%2C+by+its+actions%2C+omissions%2C+or+negligence+in+procuring%2C+compiling+or+providing+the+information+contained+in+this+site%2C+including+without+limitation%2C+liability+with+respect+to+any+use+of+this+site%2C+or+the+information+contained+herein.+Reliance+on+the+information+contained+on+this+site%2C+is+solely+at+your+own+risk.+The+information+may+change+or+be+altered+at+any+time.%0D%0A%0D%0A%3D%3D%3D+PRIVACY+STATEMENT+AND+POLICY+%3D%3D%3D%0D%0A%0D%0AInformation+Collected+By+Use+of+This+Site%3A%0D%0A%0D%0APersonally+identifiable+information+which+may+be+collected+by+use+of+this+site+includes+IP+numbers%2C+date%2Ftime%2C+stamps%2C+methods%2C+path%2C+status+code+and+size+of+request.+The+information+that+is+available+from+governmental+web+sites+is+subject+to+these+principles+and+policies%3A%0D%0A%0D%0AAccess+to+personally+identifiable+information+in+public+records+at+state+and+local+levels+of+government+in+Iowa+is+controlled+primarily+by+Chapter+22+of+the+Code+of+Iowa.+Information+generally+available+under+Chapter+22+-+and+not+made+confidential+elsewhere+in+the+Code+of+Iowa+-+may+be+posted+for+electronic+access.%0D%0A%0D%0APersons+concerned+with+regard+to+information+about+them+should+contact+the+custodian+of+the+record%2C+which+typically+is+the+state+agency+or+other+governmental+entity+that+collects+and+maintains+the+information.%0D%0A%0D%0AThe+information+collected+should+only+be+that+necessary+to+provide+the+information+or+services+sought+by+a+requester%2C+just+as+a+person+might+provide+such+information+when+visiting+a+governmental+office+in+person.%0D%0A%0D%0AThe+information+collected+is+subject+to+the+same+controls+and+uses+as+that+collected+by+governmental+offices+visited+in+person%2C+again+subject+to+the+access+and+confidentiality+provisions+of+Chapter+22%2C+or+to+other+applicable+sections+of+the+Code+of+Iowa.+You+do+not+have+to+provide+personal+information+to+visit+the+web+sites+or+download+information.+The+IP+%28Internet+Protocol%29+numbers+of+computers+used+to+visit+these+sites+are+noted+as+part+of+our+statistical+analysis+on+use+of+our+web+sites+and+how+to+better+design+services+and+facilitate+access+to+them.+No+marketing+databases+are+created+nor+are+any+commercial+uses+made+of+any+such+data.+Government+agencies+may+request+personally+identifiable+information+from+you+in+order+to+provide+requested+services%2C+but+such+information+is+handled+as+it+would+be+on+an+in-person+visit+to+a+government+office.%0D%0A%0D%0AVarious+other+web+sites+may+be+linked+through+this+site.+Private+sector+sites+are+not+subject+to+Chapter+22.+Visitors+to+those+sites+may+wish+to+check+their+privacy+statements+and+be+cautious+about+providing+personally+identifiable+information.%0D%0A%0D%0ALinks+to+Other+Sites%3A%0D%0A%0D%0AThis+site+has+links+to+other+web+sites+as+a+convenience+to+our+customers.+These+links+may+be+operated+by+other+government+agencies%2C+nonprofit+organizations%2C+and+private+businesses.+When+you+use+one+of+these+links+you+are+no+longer+on+this+site+and+this+Privacy+Statement+and+Policy+will+not+apply.+When+you+link+to+another+web+site%2C+you+are+subject+to+the+privacy+policy+of+the+new+site.%0D%0A%0D%0AWhen+you+follow+a+link+to+one+of+these+sites+neither+the+State+of+Iowa%2C+nor+any+agency%2C+officer%2C+or+employee+of+the+State+warrants+the+accuracy%2C+reliability%2C+or+timeliness+of+any+information+published+by+the+external+sites%2C+nor+endorses+any+content%2C+viewpoints%2C+products%2C+or+services+linked+from+these+systems%2C+and+cannot+be+held+liable+for+any+losses+caused+by+use+of+or+reliance+on+the+accuracy%2C+reliability+or+timeliness+of+the+information.+Any+person+or+entity+that+relies+on+any+information+obtained+from+these+systems+does+so+at+his+or+her+own+risk.%0D%0A%0D%0A%3D%3D%3D+WARRANTIES+%3D%3D%3D%0D%0A%0D%0ATHE+INFORMATION+CONTAINED+ON+THIS+WEB+SITE+IS+PROVIDED+%22AS+IS%22+WITHOUT+WARRANTY+OF+ANY+KIND%2C+EITHER+EXPRESSED+OR+IMPLIED%2C+INCLUDING%2C+BUT+NOT+LIMITED+TO%2C+THE+IMPLIED+WARRANTIES+OF+MERCHANTABILITY%2C+FITNESS+FOR+A+PARTICULAR+PURPOSE%2C+OR+NON-INFRINGEMENT.+THE+STATE+OF+IOWA+ASSUMES+NO+RESPONSIBILITY+FOR+ERRORS+OR+OMISSIONS+IN+THIS+PUBLICATION+OR+OTHER+DOCUMENTS+WHICH+ARE+REFERENCED+BY+OR+LINKED+TO+THIS+PUBLICATION.+REFERENCES+TO+CORPORATIONS%2C+THEIR+SERVICES+AND+PRODUCTS%2C+ARE+PROVIDED+%22AS+IS%22+WITHOUT+WARRANTY+OF+ANY+KIND%2C+EITHER+EXPRESSED+OR+IMPLIED.+IN+NO+EVENT+SHALL+THE+STATE+OF+IOWA+BE+LIABLE+FOR+ANY+SPECIAL%2C+INCIDENTAL%2C+INDIRECT+OR+CONSEQUENTIAL+DAMAGES+OF+ANY+KIND%2C+OR+ANY+DAMAGES+WHATSOEVER%2C+INCLUDING%2C+WITHOUT+LIMITATION%2C+THOSE+RESULTING+FROM+LOSS+OF+USE%2C+DATA+OR+PROFITS%2C+WHETHER+OR+NOT+ADVISED+OF+THE+POSSIBILITY+OF+DAMAGE%2C+AND+ON+ANY+THEORY+OF+LIABILITY%2C+ARISING+OUT+OF+OR+IN+CONNECTION+WITH+THE+USE+OR+PERFORMANCE+OF+THIS+INFORMATION.+THIS+PUBLICATION+COULD+INCLUDE+TECHNICAL+OR+OTHER+INACCURACIES+OR+TYPOGRAPHICAL+ERRORS.+CHANGES+ARE+PERIODICALLY+ADDED+TO+THE+INFORMATION+HEREIN%3B+THESE+CHANGES+WILL+BE+INCORPORATED+IN+NEW+EDITIONS+OF+This+PUBLICATION.THE+STATE+OF+IOWA+MAY+MAKE+IMPROVEMENTS+AND%2FOR+CHANGES+IN+THE+PRODUCT%28S%29+AND%2FOR+THE+PROGRAM%28S%29+DESCRIBED+IN+THIS+PUBLICATION+AT+ANY+TIME.%0D%0A%0D%0A&captcha={0}",
                    HtmlUtils.PrepareParam(captchaResp.Replace(" ", ""))),
                Rpm);

            if (html.Contains("this user agreement before you can access some areas of this site"))
                throw new Exception("Request was redirected to disclaimer page. " + DmConsts.ExcNeedRestart);

            File.Delete(captchaPath);
            return html;
        }

        private void Searching()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }

            if (Params.GetIntegerParam(DmConsts.EndPositon) > StopParam)
                Params.SetIntegerParam(DmConsts.EndPositon, StopParam);

            GetCookies();
            for (int indSearchParam = Params.GetIntegerParam(DmConsts.LastPosition); indSearchParam <= Params.GetIntegerParam(DmConsts.EndPositon); indSearchParam++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastPosition, indSearchParam);
                UpdateProgress();

                WriteLog(string.Format("Process search param [{0}][{1}].",
                    Params.GetIntegerParam(DmConsts.LastPosition),
                    _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)]));

                NavigateByPage();
            }
        }

        private void NavigateByPage()
        {
            string url = string.Format(_searchUrl, _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)], Params.GetIntegerParam(DmConsts.LastPage));
            string referer = string.Format(_searchUrl, _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)], Params.GetIntegerParam(DmConsts.LastPage));

            string html = GetRedirectRequest.RefererWithCookies(url, referer, Rpm);
            do
            {
                if (Ct.IsCancellationRequested)
                    break;

                ParseResultPage(html);

                referer = url;
                url = RegExpr.Parse(html, @"(?si:<a class=""button"" href=""([^""]+)"">\s*<span>\s*<span>next\s*>\s*</span>\s*</span>\s*</a>)");

                if (string.IsNullOrEmpty(url))
                    break;

                Params.SetIntegerParam(DmConsts.LastPage, Params.GetIntegerParam(DmConsts.LastPage) + 1);

                WriteLog(string.Format("go to result [{2}]  page with search param [{0}][{1}]",
                   Params.GetIntegerParam(DmConsts.LastPosition),
                   _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)],
                   Params.GetIntegerParam(DmConsts.LastPage)));

                html = GetRedirectRequest.RefererWithCookies(url, referer, Rpm);
            } while (true);


            Params.SetIntegerParam(DmConsts.LastPage, 1);
        }

        private void ParseResultPage(string html)
        {
            html = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(html.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));

            if (!RegExpr.IsMatch(html, @"(?si:You are viewing page)") || !RegExpr.IsMatch(html, @"(?si:This site does not contain the entire list)"))
                throw new Exception(DmConsts.ExcNeedRestart);

            if (RegExpr.IsMatch(html, @"(?si:this user agreement before you can access some areas of this site)"))
                throw new Exception("Request was redirected to disclaimer page. " + DmConsts.ExcNeedRestart);

            MatchCollection mc = RegExpr.ParseMatchCollection(html, @"(?si:document.location.href=\x27http://www.iowasexoffender.com/registrant/([^/]+)/)");
            WriteLog(String.Format("found [{0}] matches", mc.Count));
            if (mc.Count == 0)
                throw new Exception(DmConsts.ExcNeedRestart + Environment.NewLine + html);

            List<string> list = new List<string>();

            foreach (Match matchLoc in mc)
                if (!list.Contains(matchLoc.Groups[1].Value))
                    list.Add(matchLoc.Groups[1].Value);

            WriteLog(String.Format("found [{0}] id[s]", list.Count));

            for (int i = Params.GetIntegerParam(DmConsts.LastRecord); i < list.Count; i++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastRecord, i);
                string lookupKey = list[i];
                string offenderHtml = GetRequest.WithCookies(string.Format(_offenderLink, lookupKey), Rpm);
                WriteLog(String.Format("ParseOffender [{0}][{1}]", Params.GetIntegerParam(DmConsts.LastRecord), lookupKey));

                if (!RegExpr.IsMatch(offenderHtml, @"(?si:<legend>Name and Location</legend>)"))
                    throw new Exception("Offenders page:" + DmConsts.ExcNeedRestart);

                if (RegExpr.IsMatch(offenderHtml, @"(?si:this user agreement before you can access some areas of this site)"))
                    throw new Exception("Request was redirected to disclaimer page. " + DmConsts.ExcNeedRestart);

                ParseOffender(offenderHtml, lookupKey);
                UpdateRecordCount(1);

            }
            Params.SetIntegerParam(DmConsts.LastRecord, 0);
        }

        private void ParseOffender(string offenderHtml, string lookupKey)
        {
            offenderHtml = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(offenderHtml.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
            using (OffendersResult or = new OffendersResult())
            {
                or.Source_State = StateName;
                or.offenderLink = string.Format(_offenderLink, lookupKey);
                or.Source_id = lookupKey;

                or.fullname = RegExpr.Parse(offenderHtml, @"(?si:Name:</th>\s*<td>([^<>]+)<)").Trim();
                or.County = RegExpr.Parse(offenderHtml, @"(?si:<th>County\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();

                string strAddress = RegExpr.Parse(offenderHtml, @"(?si:<th>Address\s*:\s*</th>(.*?)</tr>\s*<tr>\s*<th>\s*County\s*:\s*</th>)").Replace("<br />", "@").Trim();
                or.Address = RegExpr.Replace(offenderHtml, @"(?si:<br\s*[/]*?>)", "@");
                or.Address = HtmlUtils.KillSpace(RegExpr.Parse(strAddress, @"(?si:<td valign=""top"">([^<>]+)<)"));
                or.Address_date = RegExpr.Parse(strAddress, @"(?si:>([^<>]+)</span>)").Replace("Updated", "").Trim();
                or.sex = RegExpr.Parse(offenderHtml, @"(?si:>Gender\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();
                or.Ethnicity = RegExpr.Parse(offenderHtml, @"(?si:>\s*Race\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();
                or.height = RegExpr.Parse(offenderHtml, @"(?si:>\s*Height\s*:\s*</th>\s*<td>([^<>\x28\x29]+))").Trim();
                or.weight = RegExpr.Parse(offenderHtml, @"(?si:>\s*Weight\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();
                or.Birthdate = RegExpr.Parse(offenderHtml, @"(?si:>\s*Birthdate\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();
                or.hair_color = RegExpr.Parse(offenderHtml, @"(?si:>\s*Hair\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();
                or.eye_color = RegExpr.Parse(offenderHtml, @"(?si:>\s*Eyes\s*:\s*</th>\s*<td>([^<>]+)<)").Trim();
                or.Risk = RegExpr.Parse(offenderHtml, @"(?si:<th>\s*Tier\s*:\s*</th>\s*<td>\s*<a href=""([^""]+)"">([^<>]+)<)", 2).Trim();

                or.marks = RegExpr.Parse(offenderHtml, @"(?si:<legend>\s*Scars, Marks, Tattoos\s*</legend>(.*?)</fieldset>)");
                or.marks = HtmlUtils.KillSpace(RegExpr.Replace(or.marks, @"(?si:<font color=[""]*?red[""]*?>[^<>]*?</font>)", "@"));            

                MatchCollection mcCrime = RegExpr.ParseMatchCollection(
                    RegExpUtils.DeleteTag(RegExpr.Parse(offenderHtml, @"(?si:<!--ROWS/COLUMNS -->(.*?)</table>)").Trim()),
                    @"(?si:<tr>\s*<td>(.*?)</td>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*</tr>)");
                foreach (Match matchLoc in mcCrime)
                {
                    string str = RegExpr.Replace(matchLoc.Groups[1].Value, @"(?si:<[/]*?strong>)", " ");
                    or.Crime += String.Format("{0};", HtmlUtils.KillSpace(str));
                    or.Crime += String.Format("{0};", HtmlUtils.KillSpace(matchLoc.Groups[2].Value.Trim()));
                    or.Crime += String.Format("{0};", HtmlUtils.KillSpace(matchLoc.Groups[3].Value.Trim()));
                    or.Crime += String.Format("{0};", HtmlUtils.KillSpace(matchLoc.Groups[4].Value.Trim()));
                    or.Crime += String.Format("{0};", HtmlUtils.KillSpace(matchLoc.Groups[5].Value.Trim()));
                    or.Crime += String.Format("{0};", HtmlUtils.KillSpace(matchLoc.Groups[6].Value.Trim()));
                    or.Crime += String.Format("{0}~", HtmlUtils.KillSpace(matchLoc.Groups[7].Value.Trim()));
                }
                if (!String.IsNullOrEmpty(or.Crime)) 
                    or.Crime = "Conviction;Date;County;Age at Offense;Victim Age;Victim Gender;Weapon Used~" + or.Crime;

                if (offenderHtml.Contains("<span>Aliases</span></a>"))
                {
                    offenderHtml = GetRedirectRequest.WithCookies(string.Format("{0}aliases/{1}/", RootUrl, lookupKey), Rpm);

                    if (!RegExpr.IsMatch(offenderHtml, @"(?si:<legend>Name and Location</legend>)")) 
                        throw new Exception("Alias page: " + DmConsts.ExcNeedRestart);

                    MatchCollection mcAka = RegExpr.ParseMatchCollection(
                        RegExpUtils.DeleteTag(RegExpr.Parse(offenderHtml, @"(?si:<!--ROWS/COLUMNS -->(.*?)</table>)").Trim())
                   , @"(?si:<tr>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*<td>([^<>]+)</td>\s*</tr>)");
                    foreach (Match matchLoc in mcAka)
                    {
                        or.AKA += String.Format("{0} ", HtmlUtils.KillSpace(matchLoc.Groups[1].Value.Trim()));
                        or.AKA += String.Format("{0} ", HtmlUtils.KillSpace(matchLoc.Groups[2].Value.Trim()));
                        or.AKA += String.Format("{0};", HtmlUtils.KillSpace(matchLoc.Groups[3].Value.Trim()));
                    }
                    or.AKA = HtmlUtils.KillSpace(or.AKA);
                }


                if (Params.GetBooleanParam(DmConsts.IsDownloadImg))
                {
                    int iImg = 0;
                    MatchCollection mcImg = RegExpr.ParseMatchCollection(offenderHtml, @"(?si:zoomPhoto\x28\x27([^\x27]+)\x27)");
                    if (mcImg.Count > 0)
                    {
                        foreach (Match m in mcImg)
                        {
                            if (iImg == 0)
                            {
                                or.Image_ID = Guid.NewGuid().ToString();
                                or.ImageLink = m.Groups[1].Value;
                                Offenders.DownloadImageWithExceptions(_imagesResultFile, String.Format("{0}\\{1}\\{2}.jpg",
                                        ResultDir, DmConsts.PhotosDir, or.Image_ID),
                                    or.ImageLink, TaskParams.SessionId);

                            }
                            else
                                Offenders.DownloadImageWithExceptions(_imagesResultFile, String.Format("{0}\\{1}\\{2}_{3}.jpg",
                                        ResultDir, DmConsts.PhotosDir, or.Image_ID, iImg),
                                    m.Groups[1].Value, TaskParams.SessionId);

                            iImg++;
                        }
                    }
                    else
                    {
                        string imgHref = RegExpr.Parse(offenderHtml, @"(?si:<strong>\s*Registrant Photo\s*</strong>\s*</p>\s*<img id=""regsitrantPhoto"" src=""([^""]+)"")");
                        if (!string.IsNullOrEmpty(imgHref))
                        {
                            or.Image_ID = Guid.NewGuid().ToString();
                            or.ImageLink = imgHref;
                            Offenders.DownloadImageWithExceptions(_imagesResultFile, String.Format("{0}\\{1}\\{2}.jpg",
                                        ResultDir, DmConsts.PhotosDir, or.Image_ID),
                                    or.ImageLink, TaskParams.SessionId);
                        }
                    }
                }
                if (or.Image_ID != "") or.Image_ID += ".jpg";

                if (or.Image_ID != "" && !File.Exists(String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID)))
                    throw new Exception(DmConsts.ExcImgNotFound);

                ResultFile.Write(or.GetValues());
            }
        }

        private void UpdateProgress()
        {
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.LastPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.LastPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.LastPosition))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.LastPosition),
                    DmConsts.StartPosition, DmConsts.LastPosition));
            UpdateProgress(((double)(Params.GetIntegerParam(DmConsts.LastPosition) - Params.GetIntegerParam(DmConsts.StartPosition))) / (Params.GetIntegerParam(DmConsts.EndPositon) - Params.GetIntegerParam(DmConsts.StartPosition) + 1));
        }
    }
}
